fetcher: Send Accept-Encoding: gzip when downloading summary

The summary file can get large, but it compresses well (something
which is not true of other files in the ostree repo which are
already compressed). By sending Accept-Encoding: gzip (and
handling the compressed results) we send a lot less data.

I set up the flathub repo (http://flathub.org/repo) to enable
gzip for the summary file (only), and the result is that the
331514 byte large summary was transferred in 122889 bytes.
On my (fast) network this decreased the time i took to do
"flatpak remote-ls flathub" by about 100msec.

This fixes https://github.com/ostreedev/ostree/issues/802

Closes: #882
Approved by: cgwalters

diff --git a/src/libostree/ostree-fetcher-curl.c b/src/libostree/ostree-fetcher-curl.c
index d1dab0959563d244de34a4f1e99ac28cabc7ae48..f483a6bbfc9073f3234e176f34bd77048e014576 100644 (file)
--- a/src/libostree/ostree-fetcher-curl.c
+++ b/src/libostree/ostree-fetcher-curl.c
@@ -762,6 +762,9 @@ initiate_next_curl_request (FetcherRequest *req,
       curl_easy_setopt (req->easy, CURLOPT_SSLKEY, self->tls_client_key_path);
     }
 
+  if ((self->config_flags & OSTREE_FETCHER_FLAGS_TRANSFER_GZIP) > 0)
+    curl_easy_setopt (req->easy, CURLOPT_ACCEPT_ENCODING, "");
+
   /* We should only speak HTTP; TODO: only enable file if specified */
   curl_easy_setopt (req->easy, CURLOPT_PROTOCOLS, (long)(CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FILE));
   /* Picked the current version in F25 as of 20170127, since

diff --git a/src/libostree/ostree-fetcher-soup.c b/src/libostree/ostree-fetcher-soup.c
index fdcbea52511675e96e3ebe6316fd9a7c4c6d23ec..1ca2e7718b7d2a31a42211b140826d197c08aa07 100644 (file)
--- a/src/libostree/ostree-fetcher-soup.c
+++ b/src/libostree/ostree-fetcher-soup.c
@@ -63,6 +63,7 @@ typedef struct {
 
   GVariant *extra_headers;
   int max_outstanding;
+  gboolean transfer_gzip;
 
   /* Our active HTTP requests */
   GHashTable *outstanding;
@@ -553,6 +554,9 @@ ostree_fetcher_session_thread (gpointer data)
                                                           SOUP_SESSION_IDLE_TIMEOUT, 60,
                                                           NULL);
 
+  if (closure->transfer_gzip)
+    soup_session_add_feature_by_type (closure->session, SOUP_TYPE_CONTENT_DECODER);
+
   /* XXX: Now that we have mirrorlist support, we could make this even smarter
    * by spreading requests across mirrors. */
   g_object_get (closure->session, "max-conns-per-host", &max_conns, NULL);
@@ -663,6 +667,7 @@ _ostree_fetcher_constructed (GObject *object)
   self->thread_closure->ref_count = 1;
   self->thread_closure->main_context = g_main_context_ref (main_context);
   self->thread_closure->running = 1;
+  self->thread_closure->transfer_gzip = (self->config_flags & OSTREE_FETCHER_FLAGS_TRANSFER_GZIP) != 0;
   self->thread_closure->tmpdir_dfd = -1;
   self->thread_closure->tmpdir_lock = empty_lockfile;
 

diff --git a/src/libostree/ostree-fetcher.h b/src/libostree/ostree-fetcher.h
index 78b29faeea0d75638cd9e3a2339556beb6288587..16cf3d7d6a7cfaea07c233095198c855dbd6ed41 100644 (file)
--- a/src/libostree/ostree-fetcher.h
+++ b/src/libostree/ostree-fetcher.h
@@ -48,7 +48,8 @@ struct OstreeFetcherClass
 
 typedef enum {
   OSTREE_FETCHER_FLAGS_NONE = 0,
-  OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE = (1 << 0)
+  OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE = (1 << 0),
+  OSTREE_FETCHER_FLAGS_TRANSFER_GZIP = (1 << 1)
 } OstreeFetcherConfigFlags;
 
 void

diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 08692c59eebfc1ce2bca48af246c007364b19510..751129e63d6caf59bc15d03279e2a010e95b57ba 100644 (file)
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -2161,6 +2161,7 @@ _ostree_repo_cache_summary (OstreeRepo        *self,
 static OstreeFetcher *
 _ostree_repo_remote_new_fetcher (OstreeRepo  *self,
                                  const char  *remote_name,
+                                 gboolean     gzip,
                                  GError     **error)
 {
   OstreeFetcher *fetcher = NULL;
@@ -2179,6 +2180,9 @@ _ostree_repo_remote_new_fetcher (OstreeRepo  *self,
   if (tls_permissive)
     fetcher_flags |= OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE;
 
+  if (gzip)
+    fetcher_flags |= OSTREE_FETCHER_FLAGS_TRANSFER_GZIP;
+
   fetcher = _ostree_fetcher_new (self->tmp_dir_fd, remote_name, fetcher_flags);
 
   {
@@ -2436,7 +2440,7 @@ repo_remote_fetch_summary (OstreeRepo    *self,
   mainctx = g_main_context_new ();
   g_main_context_push_thread_default (mainctx);
 
-  fetcher = _ostree_repo_remote_new_fetcher (self, name, error);
+  fetcher = _ostree_repo_remote_new_fetcher (self, name, TRUE, error);
   if (fetcher == NULL)
     goto out;
 
@@ -2544,7 +2548,7 @@ static gboolean
 reinitialize_fetcher (OtPullData *pull_data, const char *remote_name, GError **error)
 {
   g_clear_object (&pull_data->fetcher);
-  pull_data->fetcher = _ostree_repo_remote_new_fetcher (pull_data->repo, remote_name, error);
+  pull_data->fetcher = _ostree_repo_remote_new_fetcher (pull_data->repo, remote_name, FALSE, error);
   if (pull_data->fetcher == NULL)
     return FALSE;